{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "mlp_pytorch.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "metadata": {
        "collapsed": true,
        "id": "jqZrGQOG2ng-",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# MultiLayer Perceptron"
      ]
    },
    {
      "metadata": {
        "id": "EB7M0gMP2ng_",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## 1. Libraries\n",
        "*Installing and importing necessary packages*\n",
        "\n",
        "*Working with **Python 3.6** and **PyTorch 1.0.1** *"
      ]
    },
    {
      "metadata": {
        "id": "Xdemin4U2nhA",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import sys\n",
        "import os\n",
        "# !{sys.executable} -m pip install http://download.pytorch.org/whl/cu80/torch-0.4.0-cp36-cp36m-linux_x86_64.whl\n",
        "# !{sys.executable} -m pip install torch torchvision matplotlib\n",
        "!{sys.executable} -m pip install https://download.pytorch.org/whl/cu100/torch-1.0.1.post2-cp36-cp36m-linux_x86_64.whl\n",
        "!{sys.executable} -m pip install torchvision matplotlib\n",
        "\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torchvision.datasets as datasets\n",
        "import torchvision.transforms as transforms\n",
        "import torch.nn.functional as F\n",
        "from torch.autograd import Variable\n",
        "\n",
        "%matplotlib inline\n",
        "import matplotlib\n",
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "\n",
        "from timeit import default_timer as timer\n",
        "\n",
        "print(\"PyTorch version: {}\".format(torch.__version__))\n",
        "cudnn_enabled = torch.backends.cudnn.enabled\n",
        "print(\"CuDNN enabled\" if cudnn_enabled else \"CuDNN disabled\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "ateauqQT2nhF",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## 2. Variables\n",
        "*Indicate the root directory where the data must be downloaded, the directory where the results should be saved and the type of RNN (conventional, LSTM, GRU) and its respective hyper-parameters*"
      ]
    },
    {
      "metadata": {
        "id": "Jt3gMpMY2nhG",
        "colab_type": "code",
        "outputId": "cd3b3fa6-02db-4cc7-d198-7cc9d10e454a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "# Make reproducible run\n",
        "torch.manual_seed(1)\n",
        "\n",
        "# Settable parameters\n",
        "params = {'root': './data/',\n",
        "          'results_dir': './results/',\n",
        "          'hidden_size': [1024, 2048],\n",
        "          'input_size': 784,  # MNIST data input (img shape: 28*28)\n",
        "          'lr': 1e-3,\n",
        "          'weight_decay': 1e-10, # 5e-4,  # 1e-10,\n",
        "          'momentum': 0.9,\n",
        "          'num_classes': 10,  # class 0-9\n",
        "          'batch_size': 128,\n",
        "          'optim_type': 'Adam',  # Options = [Adam, SGD, RMSprop]\n",
        "          'criterion_type': 'CrossEntropyLoss', # Options = [L1Loss, SmoothL1Loss, NLLLoss, CrossEntropyLoss]\n",
        "          'epochs': 15,\n",
        "          'save_step': 200,\n",
        "          'use_cuda': True,\n",
        "         }\n",
        "\n",
        "# GPU usage\n",
        "print(\"GPU: {}, number: {}\".format(torch.cuda.is_available(), torch.cuda.device_count()))\n",
        "device = torch.device('cuda') if params['use_cuda'] and torch.cuda.is_available() else torch.device('cpu')\n",
        "\n",
        "# Ensure results directory exists\n",
        "if not os.path.exists(params['results_dir']):\n",
        "    os.mkdir(params['results_dir'])"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "GPU: True, number: 1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "BamOukfB2nhL",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## 3. Dataset\n",
        "\n",
        "*Normalizing between (0.1307, 0.3081): global mean and standard deviation of the MNIST dataset*"
      ]
    },
    {
      "metadata": {
        "id": "jpbW8mtb2nhN",
        "colab_type": "code",
        "outputId": "9d5d2d71-86ef-4184-d1d5-55c13b1b8537",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "# Get train and test datasets\n",
        "# trans = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5,), (1.0,))])\n",
        "trans = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,))])\n",
        "# trans = transforms.Compose([transforms.ToTensor()])\n",
        "mnist_train = datasets.MNIST(\n",
        "    root=params['root'],  # directory where the data is or where it will be saved\n",
        "    train=True,  # train dataset\n",
        "    download=True,  # download if you don't have it\n",
        "    transform=trans)  # converts PIL.image or np.ndarray to torch.FloatTensor of shape (C, H, W) and normalizes from (0.0, 1.0)\n",
        "mnist_test = datasets.MNIST(root=params['root'], train=False, download=True, transform=trans)  # transforms.ToTensor()\n",
        "print(\"MNIST Train {}, Test {}\".format(len(mnist_train), len(mnist_test)))\n",
        "\n",
        "# Dataloader: mini-batch during training\n",
        "mnist_train_dataloader = torch.utils.data.DataLoader(dataset=mnist_train, batch_size=params['batch_size'], shuffle=True)\n",
        "mnist_test_dataloader = torch.utils.data.DataLoader(dataset=mnist_test, batch_size=params['batch_size'], shuffle=True)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "MNIST Train 60000, Test 10000\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "QhKhpBRaU4b4",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "*Dataset examples*"
      ]
    },
    {
      "metadata": {
        "id": "ikWHBC-2U6-B",
        "colab_type": "code",
        "outputId": "ee7d39d0-a513-4361-d3ea-83b720b2a377",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 140
        }
      },
      "cell_type": "code",
      "source": [
        "# Plot examples\n",
        "examples = enumerate(mnist_test_dataloader)\n",
        "batch_idx, (example_data, example_targets) = next(examples)\n",
        "\n",
        "fig, axes = plt.subplots(nrows=1, ncols=4)\n",
        "for i, ax in enumerate(axes.flat):\n",
        "  ax.imshow(example_data[i][0]) \n",
        "  ax.set_title('{}'.format(example_targets[i]))\n",
        "  ax.set_xticks([])\n",
        "  ax.set_yticks([])\n",
        "  plt.tight_layout()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAB7CAYAAAAhbxT1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAADzhJREFUeJzt3X+wlVPfx/HPKf2gRHFOftSIcBXR\ndJD4A49GzfO4yaj7jsFDJPlZzTMaJMN4ECNTmUJ+FXfRmTQ43OE2UuE4JkVp6kqlntL4TT+oker5\nA8t3XZ192uecffa19j7v11/f1Vp778V19lnnWmtd31WyZ88eAQAQmmZpdwAAgJowQAEAgsQABQAI\nEgMUACBIDFAAgCAxQAEAgrRf2h3ItSiKBkn63+Q/S2oXx/HWFLqEGkRRdISk6ZKOk7RF0k1xHC9I\nt1f4E9+j8EVR1EXS55LWmH/+KI7j/06nR7lXdANUHMezJc3+sxxF0T8kDeZLFZzpkubGcXxeFEX/\nIekmSQxQgeB7VDC+jOO4W9qdaCxFN0BZURS11u9/Bf5n2n3BX6Io6izpFEn/JUlxHM+TNC/VTiEj\nvkdIS1EPUJKukfR+HMdr9tkS+dRT0heSxkVR9DdJX0kaGcfxknS7hQz4HoWrXRRFL0vqJmmdpFFx\nHK9It0u5U7SbJKIoaibpfyQ9nHZfsJeDJZ0kaUEcx5Gkf0qaE0VRsf/BVHD4HgVtq6SZkkZKOkHS\nvyW9Ukzfo6IdoCSdIWlbHMfL0+4I9rJZ0tdxHL/yR/kpSR0kHZ9el5AB36NAxXH8fRzHN8VxvC6O\n492SHpHUUUX0PSrmAepvkv6VdidQo/WSDvzjr3PFcbxH0m5Ju1LtFWrC9yhQURS1j6Lo6MQ/N5e0\nM43+NIZiHqB6Siqaudgis0zSJklDJSmKor9L+lH+dlmEge9RuE6T9E4URaV/lK+V9H+S1qbXpdwq\n5gGqk35ffEdg/rhjGiRpaBRFa/X7Gsff4zj+Ld2eoQZ8jwIVx/FbkqZIej+KopWSBksaGMdx0cxE\nlHAeFAAgRMV8BwUAKGAMUACAIDFAAQCCxAAFAAjSvp44ZgdFekrq0JbrlJ5srxPXKD1co/DVeI24\ngwIABIkBCgAQJAYoAECQGKAAAEFigAIABIkBCgAQJAYoAECQGKAAAEFigAIABIkBCgAQJAYoAECQ\nGKAAAEFigAIABGlf2cyBepk7d65XjqLIxWVlZS5u27Zt3voEoLBwBwUACBIDFAAgSEzxoU4qKyu9\n8rx581y8ZcsWF0+fPt1r16JFCxc3b97cxcuWLfPadenSJRfdBJAF+51dtGiRV1dVVZXxdU899ZSL\n169f7+I2bdp47VavXu3ijh071rl/3EEBAILEAAUACBJTfKiTlStXeuXZs2e7+Msvv8z4urPOOsvF\nRx99tIsff/xxr924ceMa2sUm6ZJLLnHxm2++6dXZ63LAAQfkrU9ouM2bN3vl+fPnu3jdunUuXrJk\nScZ2e/bs8epKSkpcvHXrVhd///33DeqrJP38889e+ccff3QxU3wAgKLBAAUACBIDFAAgSDlZgxo7\ndqxXPuecc1zct2/fXHwEUvTVV1+5eMKECV7dN9984+Lrr7/exWPGjPHadejQwcX77ffXj91vv/2W\ns342ZcuXL3dxct3Cfgdr2zqMMMycOdPFkydP9uoa8/q1bt3aK/fs2dPFn332mVc3aNAgF+/YscPF\nvXr18to19LER7qAAAEFigAIABKkkuQUxodZK9yZm26IkHX744S62TycfccQRdepcaOxU14wZM7y6\nG2+80cXJW+V6Ktl3Eyer65StZLaIoUOHuvi7777z6uy03qRJk1zcrFmT+dsn2+uU02uU9OCDD7r4\n9ttvz9iue/fuLk5Ow/bv37/B/Tj44INdbDOGpCyIa1Qbuz27U6dOLk5O12ZiH92QpC+++MLFZ555\npld36aWXuri8vNzFJ554oteuXbt2WX12jtR4jZrMbxEAQGFhgAIABIkBCgAQpJxsM2/fvr1X/vrr\nr11cUVHhYrtOI/kZrkOxfft2FydTxtj1Fru9WpI2bdrk4vHjxzdS7/LDZiiX9l53suw6RhNadwpO\naWlpxjq7Jrp27VoXX3755Tnvh12btWsdqN2qVatcXNu6k11fvOaaa1xs160kadeuXS5O/p61j3mE\njt8oAIAgMUABAIKUk23mn3zyiVc+5ZRTamw3bNgwr3zHHXe42G5PlaQDDzwwm4/Omr1tXrFihVc3\nZ84cF7/22msujuM46/fv1q2bi+1T/Q2Q2jbz/fff3yv/+uuvLrbTnJI0ceJEFwe0rTifgtjCbJ/0\nP/nkk706O1VttxLbn3VJOumkk1ycPHjOsr8zbNYYSercubOLly5duo9e500Q16g21dXVLj7jjDNc\nnJw2t79re/To0fgdyx+2mQMACgcDFAAgSDmZ4tu9e7dXvvLKK138wgsv/PVmic+yU0ktW7b06lq1\nauXiY445xsW//PKL187uZLGSh2/ZJKdbtmyp8TUN8cADD7h49OjRuXjLvE7x/fDDDy62mUAkP6Hr\nxo0bvbpk23yxPwfJ3aF2qip5QJ892C85rVxPQUwfTZkyxcX33HOPV2d31eZacveg/ewbbrih0T63\njoK4RlYySXLXrl1dvGHDBheXlZV57Ww2myLDFB8AoHAwQAEAgsQABQAIUk4eKU5uhXz++edd3Lt3\nbxfff//9XjubjcFmcEhKZm2wRowYkXU//5Rct7LbbWubr7fbqKdNm+bVDRw4sM79CMm4ceNcXNsh\ngjlat2mwp59+2sXJjB/PPfdcxtdNnTrVxTZjxkEHHZTD3qUrebpAY0p+9/P52YXMZi+X/HUna/bs\n2fnoTrC4gwIABIkBCgAQpEbPGnjzzTe7+Pzzz/fqZs6c6WK7DVzyE1zarcyLFy/22mXKXnDrrbd6\nZfvZkydP9uoybTtPJlW0GSjs1vdisH79ehcfddRRXl1ya3la7FTvggULXJyclrWPKNiD/CRp5MiR\nLn711VddfMUVV+Ssn2no27eviz/44ING/Syb2NQ+noDs2Z/fJLt132bmkKSdO3e6OMRk27nGHRQA\nIEgMUACAIOUkk0Rjs2fY2HOXpMy7ymxSTMnPaJHtdM4zzzzjlW2GjDzIayYJu/uqe/fuXt3q1atd\nnDyrJplYNpeS51D17NnTxbU9UW/P4xo+fLhXd+yxx7rYTh2/8847Xrs6JCsOLktBY/v0009d3KtX\nL6/OTp8nEwunKLhrVFlZ6ZUHDBiQ1etsItlHH33UxeXl5bnpWHrIJAEAKBwMUACAIDFAAQCCVBCH\n09st3dlu706urVVUVGT1Ops92Ga+LnY2G/udd97p1dn/l8kt+bleg7KHIybXoDKtO9l5eUkaMmSI\ni+3jCpK/LnLXXXe5eNasWV67oUOHZtnjpuett95KuwsF77TTTvPK9me4qqoq4+ts3amnnuri008/\n3Wt36KGHujh5qKTNvpN8lCY03EEBAILEAAUACFLY93cNkEwwm9zWaXXo0MHFCxcudLHNSFDsbrvt\nNhe//PLLXt3HH3/s4vvuu8+rswmA27Zt2+B+2ES1L730UsZ2o0aNcvHYsWO9Opv41U4ZStL8+fNr\nfL/GPNQvVDt27PDKNoHpIYcckvF1NrNIcip98ODBOepdcTvssMO88rvvvuvie++918UfffSR1+7D\nDz90sZ1ur66uzvhZr7/+uldet26dix955BEXh5iZgjsoAECQGKAAAEFigAIABKkgUh3Vxy233OKV\nbQqWdu3aeXX28LuLL764cTuWvbymOqpN165dXWznryXphBNOcPHcuXNd3L59e69dmzZt6vy5NsO6\n5D9i0K1bNxcvWbLEa9eyZcuM72nX0Ow2c7stV6rTmlRwaXSyNWzYMK9sTxewh3pedtllXruJEye6\n2KYQk/zDSmtbI7HrIjblVPI9bHqrBijYa5T07bffunjbtm0unjNnjtfuiSeecLFNVZZkH7tInvKQ\nZ6Q6AgAUDgYoAECQimqKz26bTU7j7dq1y8VPPvmkV3f11Vc3bsfqJ5gpPpvBoU+fPl7dhg0banzN\n8ccf75UvvPDCOn+unc6QpOnTp9fY7qKLLvLKNmN50tKlS11sMyLYKRGpTpkkCnb6KPn/d/To0S7O\n9P+6vuzWf8mfQkxmDLngggtcPHDgwFx8fMFeo/r66aefXHz33Xd7dZMmTXKxnXrfunVro/erFkzx\nAQAKBwMUACBIRZVJYsqUKS62U3pJzZs3z0d3ioZ96t0+yS75h6Y99NBDLl61apXX7uGHH26k3u2d\n+aI25513nouPPPJIF/fv3z+nfSoEpaWlXtlOfdvko9OmTfPa2QMLk+8xdepUF9uDL5MHQNrsLcg9\ne5Br586dM7br169fPrpTb9xBAQCCxAAFAAgSAxQAIEgFv83cbnO2mQZ2796d8TXJp64HDBiQ+441\nXDDbzLP19ttvu3jevHle3WOPPVbn9xs5cqRXfu+991y8aNEiF1977bVeO5vJ/uyzz/bqli1b5uLx\n48fXuU81aHJbmO11WblypVf3xhtv5Ls72QjiGq1Zs8bFyewn9lGJ+hwiuHPnTq/87LPPunj48OEZ\nX2czp48ZM6bOn5tDbDMHABQOBigAQJAKforPZo/o3bu3i5cvX57xNZ9//rlXtlODASm4Kb4mKojp\no3waMWKEi1esWOHV2ewcAUnlGm3atMkr20cZkr+frrrqKhfbxzVsFhfJT8C7YMECF8+aNctrZx8F\nSP6O79Kli4vt9WvduvVe/w15xBQfAKBwMEABAILEAAUACFLBpzpq0aKFi5MZzC07v2pT3ACom5KS\nv5YLmjXjb9xMNm/e7JVrWxe36aRmzJjh4mTKttoen8kkeXjoqFGjXJzyutM+8dMFAAgSAxQAIEgF\nP8W3ceNGF1dVVWVsZw9Ia9WqVaP2CWgqFi5c6JUXL17s4vLy8nx3JygdO3b0yp06dXKx/b2VlMwK\nkQ2bvVySzj33XBdfd911Xp3N6B867qAAAEFigAIABKngp/iyNWTIkLS7ABSd7du3e+Xq6moXN/Up\nvuShjBUVFS5+8cUXvbrKykoX9+jRw8XHHXec185mvbHTeMnpxOTOvULFHRQAIEgMUACAIDFAAQCC\nVPBrUGVlZS7u16+fi+3hdsl2AJBvffr0qTGWpAkTJuS7OwWBOygAQJAYoAAAQSr4Awstu+V127Zt\nXl1paWm+u9NQHFhYGJrcgYUFiGsUPg4sBAAUDgYoAECQGKAAAEEqqjWoIsMaVGFgfSN8XKPwsQYF\nACgcDFAAgCDta4oPAIBUcAcFAAgSAxQAIEgMUACAIDFAAQCCxAAFAAgSAxQAIEj/D49OSB5oVIuT\nAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 4 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "metadata": {
        "id": "dWCCngwF2nhS",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## 4. The Model: MLP\n",
        "$y_t = \\sigma(W x_t)$"
      ]
    },
    {
      "metadata": {
        "id": "N2UAizM92nhT",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "class MLP(nn.Module):\n",
        "    def __init__(self):\n",
        "        super(MLP, self).__init__()\n",
        "        self.input_size = params['input_size']\n",
        "        self.hidden_size = params['hidden_size']\n",
        "        self.mlp = nn.Sequential(\n",
        "            nn.Linear(self.input_size, self.hidden_size[0]),\n",
        "            nn.Linear(self.hidden_size[0], self.hidden_size[1]),\n",
        "            nn.Linear(self.hidden_size[1], params['num_classes'])\n",
        "        )\n",
        "        self.softmax = nn.LogSoftmax()  # nn.ReLU()  # nn.LogSoftmax()  # Softmax()\n",
        "        \n",
        "    def forward(self, x):           \n",
        "        out = self.mlp(x)\n",
        "        out = self.softmax(out)\n",
        "        return out"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "cw-AqMCfOgPN",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "*Instantiate model and optimizer*"
      ]
    },
    {
      "metadata": {
        "id": "9T6W1JeOOlTW",
        "colab_type": "code",
        "outputId": "116e71fc-f672-4a88-8b4a-15debb207249",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "# Instantiate model\n",
        "model = MLP()\n",
        "\n",
        "# Transfer model to device (CPU or GPU according to your preference and what's available)\n",
        "model = model.to(device)\n",
        "\n",
        "# Loss criterion\n",
        "if 'CrossEntropyLoss' in params['criterion_type']:\n",
        "  criterion = nn.CrossEntropyLoss()\n",
        "elif 'L1Loss' in params['criterion_type']:\n",
        "  criterion = nn.L1Loss()\n",
        "elif 'SmoothL1Loss' in params['criterion_type']:\n",
        "  criterion = nn.SmoothL1Loss()\n",
        "else:  # NLLLoss\n",
        "  criterion = nn.NLLLoss()  \n",
        "\n",
        "# Optimizer\n",
        "if 'Adam' in params['optim_type']:\n",
        "  optimizer = torch.optim.Adam(model.parameters(), lr=params['lr'], weight_decay=params['weight_decay'])\n",
        "elif 'SGD' in params['optim_type']:\n",
        "  optimizer = torch.optim.SGD(model.parameters(), lr=params['lr'], weight_decay=params['weight_decay'], momentum=params['momentum'])\n",
        "elif 'RMSprop' in params['optim_type']:\n",
        "  optimizer = torch.optim.RMSprop(model.parameters(), lr=params['lr'], weight_decay=params['weight_decay'], momentum=params['momentum'])\n",
        "\n",
        "# New results dir based on model's parameters\n",
        "res_dir = params['results_dir'] + 'mlp_{}_{}_lr{}_weight{}_trainSize_{}_testSize_{}/'.\\\n",
        "    format(params['criterion_type'], params['optim_type'], params['lr'], params['weight_decay'], len(mnist_train), len(mnist_test))\n",
        "\n",
        "if not os.path.exists(res_dir):\n",
        "    os.mkdir(res_dir)\n",
        "\n",
        "print(\"res_dir: {}\".format(res_dir))\n",
        "log_file = open(res_dir + 'log.txt', 'w')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "res_dir: ./results/mlp_CrossEntropyLoss_Adam_lr0.001_weight1e-10_trainSize_60000_testSize_10000/\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "t7ZCKLry2nhV",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## 5. Train"
      ]
    },
    {
      "metadata": {
        "id": "qaLuhdcu2nhW",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "start_timer = timer()\n",
        "\n",
        "loss_arr = []\n",
        "train_acc_arr = []\n",
        "first_time = True\n",
        "total_num_steps = len(mnist_train_dataloader)\n",
        "\n",
        "# model.train()\n",
        "model.zero_grad()\n",
        "optimizer.zero_grad()\n",
        "for e in range(1, params['epochs']+1):\n",
        "    for i, (img, label) in enumerate(mnist_train_dataloader):\n",
        "        img = Variable(torch.squeeze(img)).to(device)\n",
        "        img = img.view(-1, params['input_size'])\n",
        "        label = Variable(label).to(device)\n",
        "        \n",
        "        # Forward\n",
        "        out = model(img)\n",
        "        loss = criterion(out, label)\n",
        "        \n",
        "        # Backward\n",
        "        optimizer.zero_grad()\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "        \n",
        "        loss_arr.append(loss.item())\n",
        "        \n",
        "        if i % params['save_step'] == 0:\n",
        "            # Train Accuracy\n",
        "            _, predicted = torch.max(out.data, 1)\n",
        "            total = label.size(0)\n",
        "            correct = (predicted == label).sum().item()\n",
        "            acc = 100 * correct / total\n",
        "            train_acc_arr.append(acc)\n",
        "            # Print update\n",
        "            perc = 100 * ((e-1)*total_num_steps + (i+1))/float(params['epochs'] * total_num_steps)\n",
        "            str_res = \"Completed {:.2f}%: Epoch/step [{}/{} - {}/{}], loss {:.4f}, acc {:.2f}, best acc {:.2f}\".format(perc, e, params['epochs'], i+1, total_num_steps, loss.item(), acc, max(train_acc_arr))\n",
        "            print(str_res)\n",
        "            # Save log\n",
        "            log_file.write(str_res)\n",
        "            \n",
        "# Save model checkpoint\n",
        "torch.save(model.state_dict(), res_dir + 'model.ckpt')\n",
        "plt.show()\n",
        "log_file.close()\n",
        "\n",
        "end_timer = timer() - start_timer\n",
        "print(\"Model took {:.4f} mins ({:.4f} hrs) to finish training with best train accuracy of {:.4f}%\".format(end_timer/60, end_timer/3600, max(train_acc_arr)))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "h9iBENdtaX_o",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "*Plot training loss curve*"
      ]
    },
    {
      "metadata": {
        "id": "zMRvLLKPX63B",
        "colab_type": "code",
        "outputId": "56fe4381-c4a6-4b18-fb5c-80d24fa50b87",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 265
        }
      },
      "cell_type": "code",
      "source": [
        "# Save training loss\n",
        "plt.plot(loss_arr)\n",
        "# plt.semilogy(range(len(loss_arr)), loss_arr)\n",
        "plt.savefig(res_dir + 'loss.png')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXIAAAD4CAYAAADxeG0DAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3XeYU1X6B/BvpjDDVIZhmKE34dCr\nSBMY7CiWVXR3dddFWfvaFXX1x1rWsrgsirKgiA3d1VUBGygKIiC9M5SD9KEMM8P0wtT8/kiZ3OQm\nuckkkxz4fp7HR3Jzk/tmcvPec99z7rkms9kMIiJSV0SoAyAiosZhIiciUhwTORGR4pjIiYgUx0RO\nRKS4qKbeYF5eqd/DZFJS4lBYWBHIcIJKpXgZa3CoFCugVrznWqxpaYkmd88p1SKPiooMdQg+USle\nxhocKsUKqBUvY22gVCInIiJXTORERIpjIiciUhwTORGR4pjIiYgUx0RORKQ4JnIiIsUpk8hPF5/B\nB9/uRlV1XahDISIKK8ok8k0yF58v/xX7jhWFOhQiorCiTCKvt94Ao66eN8IgInKkTCI3we00A0RE\n5zRlErkdG+RERBrqJXIiItJQLpGb2SQnItJQJpGbbCVy5nEiIg11EnmoAyAiClPKJHIbNsiJiLTU\nSeQmtsmJiPSok8itzGySExFpKJPI2R4nItKnTCJvwCY5EZEjdRK5tUnO0goRkZYyiZylFSIifcok\nciIi0qdMIjdx+CERkS5lErkNS+RERFrqJXL2dhIRaSiTyFlZISLSp0wiJyIifUzkRESKUyaRs7JC\nRKQvyshKQohpAEZb139ZSrnA4blLALwEoA7AYinlC8EIlIiI9HltkQshxgHoK6UcAeAKAK85rTIT\nwA0ARgG4TAjRO+BROuCgFSIiLSOllZUAbrT+uwhAvBAiEgCEEF0BFEgps6WU9QAWA7g4KJFy2AoR\nkS6vpRUpZR2AcuvDybCUT+qsjzMA5Dmsngugm6f3S0mJQ1RUpM+BJibEAACSkmKRlpbo8+tDhbEG\nB2MNHpXiZawWhmrkACCEuBaWRH6Zh9W8NpsLCyuMblKjtKwKAFBcUom8vFK/3qOppaUlMtYgYKzB\no1K851qsng4ERjs7LwfwNIArpJTFDk+dgKVVbtPOuoyIiJqIkc7OZACvApggpSxwfE5KeRhAkhCi\nsxAiCsAEAEuDESgr5ERE+oy0yH8LoBWA/wkhbMuWA9gppVwI4B4A/7Uu/1RKuS/gURIRkVtGOjvf\nBvC2h+dXAhgRyKA84vBDIiINZa7sZG2FiEifOomciIh0KZfIWVkhItJSJpGzskJEpE+ZRE5ERPrU\nS+SsrRARaSiTyE2cNIuISJcyiZyIiPQpl8jNrK0QEWkol8iJiEiLiZyISHHKJXLe6o2ISEuZRM4x\nK0RE+pRJ5EREpI+JnIhIceokctZWiIh0qZPIiYhIl3KJnKNWiIi0lEnkJtZWiIh0KZPIiYhIn3KJ\nnHOtEBFpKZPIOYstEZE+ZRI5ERHpUy+Rs7JCRKShXiInIiINJnIiIsUpl8hZWSEi0lImkXPUChGR\nPmUSORER6WMiJyJSnDKJnHOtEBHpUyaRExGRPuUSuZnz2BIRaaiTyFlZISLSpU4iJyIiXcolchZW\niIi0ooysJIToC+BLADOklG86PXcYQDaAOuuiW6SUxwMYIwBWVoiI3PGayIUQ8QDeALDMw2rjpZRl\nAYuKiIgMM1JaqQJwJYATQY7FGNZWiIg0vLbIpZS1AGqFEJ5WmyOE6AxgNYCnpJRu021KShyioiJ9\njRNJScUAgITEWKSlJfr8+lBhrMHBWINHpXgZq4WhGrkXUwF8B6AAwCIANwD43N3KhYUVfm2kpLQS\nAFBWegZ5eaV+vUdTS0tLZKxBwFiDR6V4z7VYPR0IGp3IpZQf2v4thFgMoB88JPLGYmWFiEirUcMP\nhRDJQojvhRDNrIvGAshqfFiuONcKEZE+I6NWhgCYDqAzgBohxEQAXwE4JKVcaG2FrxNCVALYiiC2\nxomIyJWRzs7NADI9PP86gNcDGBMREflAuSs7wUmziIg0lEnkvNUbEZE+ZRI5ERHpUy6Rs7BCRKSl\nXCInIiItJnIiIsUpl8g5aIWISEuZRG7isBUiIl3KJHIiItLHRE5EpDhlEjkLK0RE+pRJ5EREpE+5\nRG7msBUiIg3lEjkREWkxkRMRKU65RM7CChGRljKJnNcDERHpUyaRExGRPvUSOWsrREQaCiVy1laI\niPQolMiJiEiPcomclRUiIi1lEjlHrRAR6VMmkRMRkT71EjnnWiEi0lAmkbOyQkSkT5lETkRE+pRL\n5CysEBFpqZPIWVshItKlTiK3+nT5fhw9VRrqMIiIwoZyiRwAnntvY6hDICIKG8okcpNDbYV1ciKi\nBsokciIi0sdETkSkOHUSOUetEBHpUieRExGRrigjKwkh+gL4EsAMKeWbTs9dAuAlAHUAFkspXwh4\nlERE5JbXFrkQIh7AGwCWuVllJoAbAIwCcJkQonfgwmvAygoRkT4jpZUqAFcCOOH8hBCiK4ACKWW2\nlLIewGIAFwc2RCIi8sRraUVKWQugVgih93QGgDyHx7kAunl6v5SUOERFRfoSIwAgOa9c8zgtLdHn\n9wgFVeIEGGuwqBQroFa8jNXCUI3cB14rIIWFFX69cUlJpeZxXl74X6aflpaoRJwAYw0WlWIF1Ir3\nXIvV04GgsaNWTsDSKrdpB50SDBERBU+jErmU8jCAJCFEZyFEFIAJAJYGIjAiIjLGa2lFCDEEwHQA\nnQHUCCEmAvgKwCEp5UIA9wD4r3X1T6WU+4IRqIl3XyYi0mWks3MzgEwPz68EMCKAMeliHici0qfM\nlZ2qtMhPni7H2qycUIdBROeQQI9aCRpVjjhPz10PAOjWPlmpoVFEpC5V8qMyLXKbM1W1oQ6BiM4R\nCiVy31+z+3ABnp67DgUlZwIfEBE1mcqqWtTX85Yy7iiUyH3P5DO/2IGTpyvw46ZjQYiIiJpCbV09\n7puxEs9/wFs8uqNQIg91BEQUClU1dQCAo6fKQhxJ+FIokTOTExHpUSiR+/9aM2/XTERnMWUSeQRb\n5ETnJDPbYV4pk8gbk8dNvC0FEZ3F1EnkTsn4i58PhCgS/9XW1ePdb/dg/7HiUIdCpAyejHunTiJ3\n+jK/XXvE8GvDpUa+fX8+Vu88iZc+2hzqUIjoLKJMIvenRh5uJZWa2vpQh0CkHNbIvVMmkftzeqXX\nEj95uhzzvtmN8jM1AYiKiCj0FErkxjL5xr25+GTZr9rXOrTMZ36xE79k5fhUmgkUNiyaRl19Pcxs\nxp3VThVWYOanW9kgs1IokRtbb/aiLCzdmI1Kh0mrHFvm5ZWWL952tZg/vlp9yOVgQeGhtq4ed0xb\ngTe+2BnqUChA9H77sxZk4YcNR/H1L4ebPJ5wpEwiD6ca+aLVh7B0YzaKy6tx4DhHoISTM9WWA/S2\n/fkhjoQCRe/kqqyyGgBwppqzjAIKJXJ/auSNaXUb8cScNXhx/maUVFQbewHP9okoCBRK5K6ZfM/h\nghBE0qC6xjIKpeIMWwXhgrVxOhcplMhdl736ybamD4Sa3Ma9uZj79S4m6TCwdV8eVmw73qTb5AVB\n3imTyN3VyDfLXPywMRuApaMrFIzuZ+FyYZJRNbV1+H7DUZSUGywdBUBJRTVO5Jdrls1elIW1u04h\nt6jS6+vV+gur540FO/Hhd7JJtxlux+9lm4/h/tdWhtWZuDKJ3N3ww1kLs/Bf6wiSUF1wE2b7WcB8\nvyEbny7fj7e+2tVk23xo5mo88856/bvBnIV/6OP55fhpa9O2cMPd0o3ZOJJTGvTtbNqbi+mfbvOp\nAbh8yzF8/MM+lJ+pxd6jhUGMzjfK3HzZyOlVuJ96Gwlv6695aJeWgNYtmgc/IC9OW2+R59xCDqTD\nOSUoLKlC13bJyD7V8OOtN5sR4XSuI7OLUFVTd1bd1Pr/3rHcrLt7+2S0T0sI+vZyCytwIr8CA7u3\nCvq2/JFTUGEf2vvukxe5PJ9fXInv12ejIgD3xP33oiwAwN6jhejbJdXQaz5aus/+73BKNwolcu+Z\n3NMt/errzS6ljcLSKiQnNAubKXILS6vs45/1duKz0fPvbzK87vtL9gIAvp5+bbDCCRnbsMlAqDeb\n8e2awxgsWqNdq3jNc0++tQ4A8Nr9FyIpvlnAthkIhaVVeMHL7dzmfLkLB0+UBHS7/g5TDqeGo0Kl\nlca9/ok5a3HP9JX2x0dzSvHorF8w75s9bl/T1F9UZQBaGYFk+/jlZ2qQX+y9Ph0WDHxlB0+UnNVX\nBGYdPI2Fqw5h6rz1btcJ9tBcf3y5+iAqq1zjcvztF5c1XX+NvzbLPHz20/4m3aYyidxIq9lT4j1d\nckZTCztgPaqv3ZWju/62X/Mx+R8/GbrgJ1BXl4XJiYGL2jozpsxeG+owAiK3qBJ//3ATnn337L2R\nr63s4KkdEogmyrG8soD2S7k7o3b8HJ5+I009NYPzlmrr6nGqsAKzFu7EkvVHm7Rhpkwi95bk1mbl\n4L3Fe+2P6xz2CqPf7ZGcUqzNsiT2z1ZYjqhLrSNiPG7bzcHAUb2XIPKLKr2Wj8oqa7y+TyAF8sCy\n53ABHnh9FY7nBe4GukdySrFs8zGfXlNorfvb6v9NZcu+PBQ00TaXbwl+5+n+48WYOm8DZlvrzIHQ\n2N3tjmkr8I//bG3Ue+QWVeLF+Ztw9JT3zlbng8brn+/AU9bSld7zwaROIvfy/Nxvdmsuy/5xU0MC\n/nnbCUPbeO79jZj7zW63w4oWrTqI219Zbui9HH3x8wH8+R8/ub0CdOu+PEyZsxZfrHB/s4y/zFiJ\nB15fhTlf+jeCpOJMDXIKKnx6TSD3w/eW7EVZZQ0WrwvcZGXPvb8RH/+wD7mFDZ8rfKqWDY7nleHN\nBTvx9DvuSx1GlJ+pwbxvdnv9HpvixiW2USWBnArBSMPB2zr7sos0j/cfK/bpwsHPlu/HgeMlmPv1\nbq/rOv8+dh3Sbqcp90VlErmvThU21HR9rQfW1tfbvyTHYXBf+VlCsc20eOC4fidNlnUH2Lwvz+17\n2E6XN+3N1Sxfm5VjaEd9fPYa/PXtdaj28Lf4eOk+zP3a/6GGpwor8N36o15aIoGvH1XVGD+9D0X/\nVEmFdaK2RnZofrPmMH7JysEbX+xodEyl5dWNuu7Cn7O1yqpaL+UG/TdtzJnhSx9t9nrhoN51Esfz\ny/HaZ9s9vs5oi7uqpg7frT0c1FKLQok8NAXkzfvy9Mc0O6mvNyPfwAUrelbtMHbGoGfuN7sNXeFq\n60Sq9lDTXLblGNbuOmVouzW1dXjmnfWa0saz723E/37aj50HtQeWk6fLkV9sKSsY/VFW1dQZ7mD1\n5RS2MXk8FKMUKqtq8cvOk6iprbMfCGwzePrq5OmGYaQvzt+Ml+b7f6cqwxfBmc2YtWAnVmw9jvtm\nrMR9M1a6XXf9Hv19T1Mjd9ryyu0nXRo3eiqralHq5ox47jf6re8dB07b696LVh1EXb32t2N0b/hy\n9SHM+nw7/vPjPu8r+0mZRO7rUXn9bmMJSc/arBzN6auRDp33Fu/BlDmuHYL1mlq9/ldfWxe4BHEs\nr8zjuO9ZC3YiO7ehTp1bVIl92UU+zyJ3OKcUJ/LL8fEP++wHMHuicRoR8vTchpKC0a/xqbfWYcrs\ntYbOpj6wXmm450ghvlp9yL78VKFrCaIxyfjpuevxz08aV4N1y01YH/+wD/O+3WM5q7P+CPz9BI7f\nA2D5DoOttLIGm/fl4cPvPV8NeiK/3NgZi84O9G8Ddfr7ZqzEgzNXe39/J2Yz8PL8zfjql8NY59zI\n8fJFvPzRFpRV1uCk9fd4LDd412Mok8ibcqz3p8t9Hzr0S5Z+h+cuPyf2KnNodeX50NKfOm8DnvFQ\ni5XZRZpTxifnrMUrH2/BjU9961ecgM5nDMBxyfb5jYyvPnSyBLsOFeDV/27VXCX51FvrcCSn1Glu\nevfbW7LuiMfT35yCCuw+7Plqvl2HCjyOdFqx7Th+1pmrpPxMDcxmM8xmM16avxlLrH0JtmR7OKfU\nXhbz5Vi0Juskvllz2PgLYCnhTPvPlkYd9P769jq8+637ob3OPvfQP+Qo2FnA+RObTA2lMee+M29T\nbpzIL8dPW3zrjPeXMok8pllkQN6nzI/T0uraOpwu9m/EQbWb+q23Cx9s26utq8cTOi19Z+vcjJwp\nLK3Cq//VtiKN12pdd9QjOaV4f8ke1DqcpTiPtnHueNYwWUbo1NTWQTbiEmfnJDP9U/3y0nPvb9SW\nENz89j78bi8+W3EAi1Yd0l/BoOmfbsOL8zfjjmk/YbN0PeX/8DtpP4NwPNV//fMdmP+9xPH8cuw/\nXozPnBLbjgOnscZNY8GTd77ZgwUrDxpat+JMLT5f/isWrDyIvUeL3O67ADSnyFXVdTjmNBopp6AC\nq3ee9Dler3xo0O0PwL0CXvRQfjJynMspqMT2A6ct6wex+1OZRB5sjvVDZw/OXI3HZ6/x6f2efW8j\njua4vwLt0MlSlx52R/VmMzbuzUVpheuBx1aXdkxmbzv1stfW1ePk6XJMmb0Ge47oJ8yaWvcJvbqm\nTndHfeGDTVi5/SSWeRni9q2bVmBBSRWmzFmLu/75s99DxZ6Z8wv+Otf4CJDjDqUmxx/Ti/M32evw\nts7x/OJK1Neb8eOmbJfk5I1jCa6u3qwZDqvH+VR/xbYThvpjamrrsftwge66jWlFf7r8V3zwbcN+\ntHlfLorLqnTXdUynr3y8BVPnbcApH0dFOTI6+sVoGt91qMBtH8C2/fmGh606zvlSUKptzNXVm93W\n3W2MDE0OBGUu0Q+m6Z9uQ2JcdMDf94Nv92CoSLM/dv6NvfLxFreX4q/JysGyzcfQo32yy3Mf/7AP\nKYkxeHOB9nZmjjvVna+u8Brf/a+t0l2+50ihSyvexjaOfYvDCBu9H9cBN5dR5xY2/grR7b/6N+St\nrLJG03dy4HgJFqw8iDuv7mP/DGYz8OdpP9nXeffJi1BVXefxjPBYbhlSkmLw8BvaxOwppTZmUqiq\nmjr885Nt+ONlPTB2UDtN2bExbb58p7POd77Zg9SkGLx67yjXlR2+9CPWMde5RZVIbxnXiAi0CkrO\nYMZn23HdhV0bNmswkzv2Azmb+bl/o36+36C9puT9JXvx/hLgtQcuRFJcaKc7MJTIhRAzAAyHZT95\nUEq50eG5wwCyAdiad7dIKZWazs15/GegbNidgw27G47ItfXGh3vZWgz73IwJ/q9OD7ivnTnuRrC4\nS+Ju+VC4jIoKzUngydPl+OA76XIWZHL6h3PLcNfhAkz/ZBsmjOys+755RZWY+u4Gt9utrqnDFuk6\nrPS59xt/Zen8pfvwv58OYPajYxsWBvjs/XRJFX49VoTu7VtolutNZTvjf9sxtGdr3HNdX/uykkZc\nUv/d+qM4nleOWQu93381mBO7eZNfdCbkidzrr0oIMRZAdynlCACTAczUWW28lDLT+p9SSbwpZR0M\n5AGj6Ydjuqtp+zLpUHRkaIaRPj13vcdSljvrrSMVHDsMyypr7InD85BTM+Yv2YNlPnR42SYGM6qx\nc6as3ZVjL7G5a+2+/NEWw++30WkooKeDnDe+HJM8dfAHWzhMnmWkeXQxgEUAIKXcAyBFCJEU1KjO\nIU8a6MjUE4p5WdzVtL9eY7yDMDLCtxZ5XRPcLMRsNuPoKeP18EfetMyZ7qmPAbCM3V/0s7HRGDaO\nQwJPni73uaXpa4fa3K9321vXnnapQCarjXtzcff0Fd5HY+ls0sgsqP5ozNQXhl8ZxHxv5FeVAcDx\n3DDPuszRHCHEaiHEK0KIMJ36KTwZueuNnsb8rmoCnBzzis5gTZbrCIUdB/Lx0ExtHd7X3+Fj//at\nk9lX63ad8nlEkm3cf01tfVCPqM7jvo3wZ7+wDZ31lCT1ZiX01+xFWaiuqceqHZ5HteidyRg9oLg7\noDlf1GOzcOVB1JvN2OtmYICXjRly1EPdvrH86ex0/ranAvgOQAEsLfcbAHzu7sUpKXGIigrMUMJz\nWWMmfaqprUd1gEsz7+hMB/zaZ66dStHR4fXdmwHdC7lsDnnolExtlYiiM6GfDvbjZb9i0lV90CIx\nxu1l96mpnm9akZaWiJgY9+kgISkWyQkxhuIxeuOPAydKcPsry9HOhxtqnDztfWTM3uMlKHXzvVS4\nuUxg495cdMhI8uuGFcktmhv+zMG6KYqRRH4C2hZ4WwD2Q6mU8kPbv4UQiwH0g4dEXqhztR01vQXL\ngne5sCdNUSoJJE+zNZ7OL0VxUej352Ubs7Fi8zHMengMIiL0D9CPzFjh8T3y8kpRVOq+cZCfX4a/\n/vsXQ/Fs2GGsm8w2R1AgZ8QEgGnz3d+s5MF/rdBdnltYiTkGOlX1FBZWIC/BWGfn1t0n/b4TlKeD\ngJHSylIAEwFACDEYwAkpZan1cbIQ4nshhO1TjAUQuHktKWiMTM8bDOE657o/Qt/F1aCu3owvfj7o\ntrTibjioTX5xpdtJ3QBgyfqjhuv1L3xg/K5P55qp8/zv/PXEa4tcSrlGCLFZCLEGQD2A+4QQkwAU\nSykXWlvh64QQlQC2wkNrnChYnVWhcP9rqzB+WMdQh2H3w6ZsxPp5BbS3G4f4Ou/7uWSzzEOn9NDe\nR9bU1ENn8vJK/d7glNlrXC5aILX0aJ/sdmw8kapMJuMdzf7ejzctLdFtK0ipS/TfefrSUIdAjcQk\nTmejUA8lVyqRR0YqFS4RUZNgZiQiUhwTORGR4pjIiYgUx0RORKQ4JnIiIsUxkRMRKY6JnIhIccol\n8owA3kqKiOhsoFwi//ufh4U6BCKisKJcInc3TScR0blKuURORERaTORERIpjIiciUhwTORGR4pjI\niYgUd9Yl8vtv6Nfo95gwsnPjAyEiaiJnXSIf1D1Nd/llQzsYfo+2rXjRERGp46xL5AAQF+N6T+nf\nXdwd7dPiDb0+OtK/G9gSEYWCkon8rccycfkF7lvY//rLKN3lLRJj7P++bGgHXNivDXp1SsGM+y9E\nlMNt5AZ2T0WfLi0DFzARURC5Nl0VEB0Vgf5dU/H9hmzd55tF67eob7m0Bz5Yshd/vFygTaq2dd6z\nYwtkHSpAlzZJiIyIwKO/HYhNe3Px70VZAY+fiCiQlGyRA0DL5FgAQLPoCNwwtqvHda8fY3k+PSUO\nU24e7JLEAQA6V/4PFq719qmTzrf/e8yAtvjbpKFIcWjpn2siGzllwn2/6dvoGC4e3L7R70GkMmUT\neXpKHP7vT+fjX/dd6HG9GzO7GRuFYnZdFGHSJqk+XVqic0YSbr1cAABG9s1Ap4xETL9Pv5TjyQW9\nWvu0/pgBbXSXP/WHwbhtfE9D79E6pTl+d9F5LsvHDmyLlknGDka3XNoDk6/q5bD9IYZelzmwre7y\nIcK3v4Mex/l3hvTQ7+z21btPXoT0MJ9p88J++vsEha/enVOC8r7KJnIA6NImCXGxUejfrRUA4Gqd\nhK2Tnz0yuWlgTr6qFx6+cQAAIHNQO7z9eCZ6dGhhf37S+J4YP6yj/XGzKPd/2o6tE3BjpmtCBYDO\nGYl498mL8PbjmXjm1obW/6TxvdC6RXPNuu3T4tG9fQuMHqCfJN9+PNOedAf3SMOU3w9Cu7QEl/XG\nDGiLD/52Bd598iKX5+68ujdaWc9+AKBvl5YY1a8Nbr1cYNzgdujaNgl3XN3b7We1aeUUu6Onb9U/\nGNx6hfD6vgAQ4eNe/O6TF+Efd4/QfW7yVb1wp/XzPBCAoaw270wZh2dvG4onbh7k8py/Cbl3F9ek\n4LhPNqWE5tFun7v72j6Nfv/R/dtgaM/Wmr4sFT1y08CgvK+SNXJnHVonYM6jYzW18ZF9M7AmKwfd\n2iYFZBuJcdGalp/zDjXGmkyXrD8KAJjzWCaSWsRh3sId9mVXjeiEb9ceQceMRN1t3DTuPPswyajI\nCJeDivPjEX0yXN7jjYdG4/7XVtnfY1S/NhjRJ8Me+7G8ciMf1254nwwM75OBIzml2JddZG+lZg5q\n5zYumwkjOwEAYptF4dLz2+PzFQc0z9sOAN3aJtuXpSTGoLC0CgAwdkBbXNCzNU6XVOH9JXtw6GSp\n7nbMjkdrE9Cncwp2HS7UXfd3F3cHAMS46UcZ5ZBU631tBXgQEWFCx3T97330gDZYvfOk29deObwT\nFq87Yn98Yb82GD+8I47kuP49nrxlMADg9leWa5af1z4Z+48V+xO6i47pCTh6qkyzrGVSDC4a3A4r\nth7HRUPaY9GqQwCAv00aig7pCbiloga/HivChj25fm3zj5cLREVGYPonWzXfbUbLOOQUVPj0XjeN\nOw//+2m/X3FM+f0giI4t8NRb65BbVIkubRKRnhKHdbtP2dd56Mb+WJOVo/tZgzV7q9qHNwfOHZy3\nXdkTr9w1HKKjb6cyZqcfry1xtUyK1Vnb1R0TeuM31pp8THSkJmFcM6ozJl/VCzdf0h3xzV2PoVcM\n66j5op0T5D3Xea8nm2DCC38ehudvv8C+zPE9650/oBv3XtcXz09ueI9OGYm41M1Y/L5dUtEsKgI3\nX9Idv3Uo3Vw/phuuH9MNVw7vhOioSDw/+QJN69PxQBTbzPJ3GiLScN3oLhjaszVMJhPiYqPRoXUC\nHrKeDQ3t2RqXD+9k7xdJTYrRfKZrR3XBAxP747nbL8CbD43WxPnITQN8up6gpZu+jzsmeD8D8eQ3\nY7oi1WF/MpuBgee1crv+xMxumsfjh3dEm9R4l321V6eGff2KYR3xh8t6NLzmgo7wh96Q3bED2+ms\nCVw3uitee2A0hvVKty/rlJGICJMJFw9pjzuvaWiZt9c5M3RnRJ8Me8Pp7uv64qZxDfvYLZc2fEZ3\n9yq4wuFMedbDYzSP01NczxQ93fMgMS4aJocfZnpKnOZzAUD/bq1w97V9XfrOfC2n+uKsSeTOIiMi\n0DrFeI3TlqhTk7UJ+5lbh+CJmwcZ3vFG9M3QlngcknF0VCRG9WuD2GZRiG0WhX/eO1K3HGQT6VQz\n6JieqFv+AICHbxqA6y7sgrjYKLRrFY/2rb3He2NmN6Qmxej+WHt0bGH4Myc0j8acxzJxyfkdcLmH\nhNE+LQE9O+mf+tt+GyaYcM2HtNNDAAAMa0lEQVSoLi4HrcS4Zpj96FjcfW0f/OXGgbhqRGfMfHA0\nXr5rBMYP64SOrRMs31PrBERHRaJD6wTExUYj2lri6tM5BX27pjps0DWGSU59Dc1jojDr4TEY1N2S\nZP/6hyG4/4Z+GNYn3eW1Uyed71JeeOCG/rqf9eqRnfHqvSPtZZC0Fs1x/w398OZDozHYQI3f1lnv\neAB75e4RePimAfbHN407Dxc5dALH6lxb4S6xaEtlJrRtZeD6C4eDiq2/ZYTT30nb5+S+QXHl8E72\nf996hcDtVzV8L/Gx0bhsaAekpzTHTZf00AwTbtsqHlGRrl/sEIdBC82d/g5XjuikeTygW6rm8w7r\nrf0MyQmWz9a5jeXsqk2q+xwzweG9F/zjatx1TeNLTO6cFaWVQLhpXDe0SGiGi4doR0DEx0b73Ko3\nqmVSLC67oANkdhF+M7qLy/PuLmDq1i4JB46XoLVDa6Jf11T0c0xUbvTt0hKDurfCuEHt0LdrKsYP\n1+7IPTq0wL7sIt2LqgJhcI80dGt3HFcO0273oRsH4JNl+zWtJWfO5RBb4kxJjMGzDmcgjqb8fhAW\nrDzo0mqKj41CbLNIDO+djhXbTgBoKI85ah4ThXt/0xeFpVVoldzw905oHo2yyhoMPK8VJmZ2Q9tW\n8fjXX0ah/EwtHn5jNQBgYPdWMJm0LWVHj/1uIEorauwtt7jYaPzl+n4uZREjnPtPbJpFRaC6th6J\nDgeZC/u1QbPoCFw9qgtG9m2D1z7bjmdvG4rC0io0j4tBj7aJmPv1buvaZjx3+1DcMW2F/fV6xQHH\nM7HoqEi8M2Wcbhnhrmv64Ju1h9G/Wyu3ZT6zQ5Jv0zLOpUETEWHCy3eNQFpaIvLySjF10vn2g8Sr\n946y//1tbGc/nR1Kmg9O7I/v1h/FBT3TseDngygurwYA3He9tl9k4thuWG8tm/zz3pH2fe5PV/RE\nv66puKCX60FdT3RUhKYlH2hM5FZxsdG4brTnYYzBEB8bba9rOjOZTGiXFo9Ipx3gwYkDsPdIoaHW\nm7OoyAjc76alCABTbh6E2tr6RnUq3X1tHxx38yONbRaFp/94vsvy7u1b4P/+5Lq8sbq1S8bjv3ft\nYIyMiMCsh8fAZDLZE7k7kRERmiQOWE6xyyprEBUVYW/BRUVGIDm+mWa9hdOuwel8/dp+VGSE7tDV\nS85vjz1HCjV/wwcm9sdbX+3SjDqKj3XfwWjz4h3DcfBkCdq3TkB8bBTKz9Sibat4+wGzf7dU+1le\nx/REe3K8fkxXLFh5EH+e0BuRERGIMJlQbzZjZN8Ml7b0W49l2s98bNzVgof1Tsew3ulYm5WjWf72\n45l49r2NOJFfjgSHz2Uk+XXOaOgHc/77Zw5sixYJMZh2zwgkxzf8rQec1woDrOWs9JZxKC6vxjWj\nOtv3+0nje2Lbr/lISYrBnEfHAtCWb5vHRGn6U/QEsIvFKybyIBveOx2LVh3CbVcaGyLo7HmdlmZC\n82ic3zM49bYIk8ntBVVGXdArHejlfb1QsyWJ1KQYnC6pCso2IiNMPrfEbr7EUvd1bJkPPK8VZj8y\nVrNe//O8n4GlJsfay4VTbh6MpRuPYtwg/Rq3owkjO+sO23U+8LhreXszrHc6qmvr8MF3EoDloPbQ\nxP74efsJXDykPdq2isfaXTk4r12yl3dy75Ih7XGztYbufCB2dPe1fbBq+wlc5lAWHDOgrf0Mzcjv\noWPrBBzNLfO6XrAwkQdZ65Q4zHtinN+nVcE8HSOLV+4egdo639pPF/RKx5erD6G/Tjnr8d8PQlPc\nWjbCZOnYNrqtDq0TMPkq/zpqB5yXiq2/5qND6wR0bWNpAV83uovfozAiIkwYO7AdBnZPs8ffqkVz\n3DC2m3V7DS1mXz3+u4GIiDAZHorZIiEGV49yLW364rHfD8IDr69CH4dx4rZ6vK0jP5hMZoOjGAIl\nL6/U7w3aTvtUoVK8jNU39WYzcgsrkZ7S3OPBtjGxfr/hKCqrapu05Ocu3qrqOuw/UYzenVJgMplQ\nX28O+Y3Qw2E/cFRxphaxMZH2en1dfT0WrTqEkX0z0L9nRqNjTUtLdPsHZ4ucyA8RJhMygnzlp6cR\nQE0tplkk+nRuGCES6iQejuJitek0MiLCfoYRbGft8EMionMFEzkRkeIMlVaEEDMADIdlRM2DUsqN\nDs9dAuAlAHUAFkspXwhGoEREpM9ri1wIMRZAdynlCACTAcx0WmUmgBsAjAJwmRCicdcvExGRT4yU\nVi4GsAgApJR7AKQIIZIAQAjRFUCBlDJbSlkPYLF1fSIiaiJGSisZADY7PM6zLiux/j/P4blcAB67\naVNS4hAV5f+4yrQ0/RnkwpVK8TLW4FApVkCteBmrhT/DDz2NO/I6Jqmw0LcpJx2F27hRb1SKl7EG\nh0qxAmrFe67F6ulAYKS0cgKWlrdNWwAn3TzXzrqMiIiaiNcrO4UQIwE8J6W8VAgxGMBMKeWFDs/v\nAnAVgGMA1gK4RUq5L4gxExGRA0OX6AshXgEwBkA9gPsADAJQLKVcKIQYA+Af1lW/kFL+M1jBEhGR\nqyafa4WIiAKLV3YSESmOiZyISHFM5EREimMiJyJSHBM5EZHilLmxhKcZGEMQS18AXwKYIaV8UwjR\nAcB8AJGwXCz1RylllRDiFgAPwTJs820p5TwhRDSA9wF0gmXGyNuklAeDGOs0AKNh+a5fBrAxHGMV\nQsRZt5UOIBbACwC2h2OsDjE3B5BljXVZuMYqhMgE8BmAXdZFOwFMC+N4bwEwBUAtgKkAdoRxrJMB\n/NFh0fmwTCA4G5ZctUNKeY913ccB3Ghd/pyUcrEQIhnAfwAkAygDcLOUssDXOJRokRuYgbEpY4kH\n8AYsP1yb5wHMklKOBrAfwO3W9aYCuARAJoCHhRAtAdwMoMh6UdWLsCTXYMU6DkBf69/tCgCvhWus\nAK4GsElKORbATQD+Fcax2jwDwPajC/dYf5ZSZlr/uz9c4xVCpAL4G4ALAUwAcG24xgoAUsp5tr+r\nNe4PYPmdPSilHAUgWQgxXgjRBcDvHD7Xv4QQkbAciFZY410A4Al/4lAikcPDDIwhUAXgSminIsgE\n8JX131/DsnMNA7BRSlkspawE8AssR+qLASy0rvujdVmwrISlBQAARQDiwzVWKeWnUspp1ocdYLlS\nOCxjBQAhRE8AvQF8a10UtrG6kYnwjPcSAD9KKUullCellHeGcazOpsJycWQXh4qBLd5xAJZIKaul\nlHkAjsCy/zjGa1vXZ6okcudZFm0zMDY5KWWtdcdxFC+lrLL+OxdAG+jPDKlZbp361yyEaBakWOuk\nlOXWh5NhmWY4LGO1EUKsgeVU86Ewj3U6gEccHodzrADQWwjxlRBitRDi0jCOtzOAOGusq4QQF4dx\nrHZCiKEAsmEpBxV6isvDctsyn6mSyJ2F851f3cXm6/KAEUJcC0si/4vBbYcsVinlSADXAPjIaXth\nE6sQ4lYAa6WUh3zcdqj+rr8CeA6WMsWfAMyDtn8snOI1AUgFcD2ASQDeQ5juB07+DEtt3uj29Zb7\nHasqidzTDIzhoMza8QU0zADpbmZI+3Jrx4xJSlkdrMCEEJcDeBrAeCllcbjGKoQYYu00hpRyGyyJ\npjQcY4VlkrhrhRDrYPkB/x/C9O8KAFLK49bSlVlKeQBADizlyXCM9xSANdYz3wMAShG++4GjTABr\nYGldp3qKy8Nyv2ePVSWRLwUwEQCsMzCekFKG00TEP8JyuztY//8dgPUAhgohWgghEmCp1a2C5bPY\n6tZXA/gpWEFZe8RfBTDBoSc8LGOFZVK2R61xpwNICNdYpZS/lVIOlVIOB/AOLKNWwjJWwDIKRAjx\nmPXfGbCMDHovTONdCuAiIUSEteMzbPcDGyFEWwBl1vp3DYC9QgjbDLHXW+NdDuAqIUQz6/rtAOx2\nitf22XymzKRZzjMwSim3hyiOIbDURzsDqAFwHMAtsJxWxcLSiXGblLJGCDERwOOwDDd6Q0r5sbWn\n+h0A3WHpOJ0kpcwOUqx3AngWgOO0wn+ybj/cYm0Oyyl/BwDNYSkFbALwYbjF6hT3swAOA/g+XGMV\nQiTC0u/QAkAzWP62W8M43rtgKQUCwN9hGTIblrFa4x0C4O9SyvHWx70BvAVLQ3m9lPIR6/L7YckV\nZgDPSCmXWQ9CH8HSii8C8AfrmbNPlEnkRESkT5XSChERucFETkSkOCZyIiLFMZETESmOiZyISHFM\n5EREimMiJyJS3P8DZc58q6idw8QAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "metadata": {
        "id": "bUXii0BC2nhZ",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## 6. Test"
      ]
    },
    {
      "metadata": {
        "id": "q3Qlsqzo2nha",
        "colab_type": "code",
        "outputId": "776414c0-e9de-46a7-f14a-54e2f8c67f0d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 382
        }
      },
      "cell_type": "code",
      "source": [
        "# Testing doesn't require the use of gradients since weights aren't being updated\n",
        "model.eval()\n",
        "with torch.no_grad():\n",
        "    correct = 0\n",
        "    total = 0\n",
        "    \n",
        "    for img, label in mnist_test_dataloader:\n",
        "        img = torch.squeeze(img).to(device)\n",
        "        img = img.view(-1, params['input_size'])\n",
        "        label = label.to(device)\n",
        "        \n",
        "        # Forward\n",
        "        out = model(img)\n",
        "        \n",
        "        # Test\n",
        "        _, predicted = torch.max(out.data, 1)\n",
        "        total += label.size(0)\n",
        "        correct += (predicted == label).sum().item()\n",
        "\n",
        "    # Accuracy\n",
        "    print('Test Accuracy: {:.4f} %'.format(100 * correct / total)) \n",
        "\n",
        "    # Show 4 test images\n",
        "    fig, axes = plt.subplots(nrows=2, ncols=2)\n",
        "    img_cpu = img.cpu()\n",
        "    label_cpu = label.cpu()\n",
        "    print(\"Label: {}\".format(label))\n",
        "    print(\"Predicted: {}\".format(predicted))\n",
        "    for i, ax in enumerate(axes.flat):\n",
        "      img_cpu_i = img_cpu[i]\n",
        "      img_cpu_i = img_cpu_i.view(28, 28)\n",
        "      ax.imshow(img_cpu_i) \n",
        "      ax.set_title('Target: {} - Prediction: {}'.format(label_cpu[i], predicted[i]))\n",
        "      ax.set_xticks([])\n",
        "      ax.set_yticks([])\n",
        "    plt.tight_layout()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:15: UserWarning: Implicit dimension choice for log_softmax has been deprecated. Change the call to include dim=X as an argument.\n",
            "  from ipykernel import kernelapp as app\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test Accuracy: 90.6600 %\n",
            "Label: tensor([7, 2, 6, 2, 7, 1, 3, 0, 8, 6, 3, 0, 4, 7, 7, 9], device='cuda:0')\n",
            "Predicted: tensor([7, 2, 6, 2, 7, 1, 3, 0, 8, 5, 5, 0, 4, 7, 7, 9], device='cuda:0')\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAVEAAAEYCAYAAADlIcXmAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xu4XPO9x/F3kAhJiMQt6ihR+QZt\nqKZSlQSNRgWlxK3auEVQh6Rt6vbkQcl5RHvkoG1oXcJDKq4hkRanJSSog0epS75F4xKaBJGIkKuc\nP9ba7fx+e/bsPfs3e2dm78/reTzymVmz5jez1+87v/WbNWt1WLduHSIi0jwbrO8GiIjUMhVREZEE\nKqIiIglUREVEEqiIiogkUBEVEUmwUak7zexa4IA87gy8B3yW56+7+7IWbBtmtivQ093nNLLcVGDP\ngps2Bx5392PLeK4vAX/P/wPYkOz1nuPuL5TV8Prrvhl4HbgceBkY6O4flFj+NHe/3sw2bMryzWzT\nscDP45uBLu7+aSWfq62poX7REbgKGEI2YPoTcLa7ry3juTYCVpNtv2vz9SwBznP3R5v3Cv617kuA\n7d19pJnNAkaX6mt1/SL/d6PLJ7RrEHAl0A1YDowp9V6XLKLufmbBit8EftDYH67CjgLWACWf092P\nK8xm9hBwczOeb5W79y1YzwnAfWa2i7uvacb6AvnG27fUMma2PfBT4PqmLJ/QljuAOwqe9/vA4Sqg\njauVfkG2He0E9AM6ALOAEcDkZjznIHdfAGBmg4FpZtbH3T9sxrrqcff9S92ffyBcAVzflOWby8w2\nAe4DjnD32Wb2XWAK8MWGHlOyiDbhCXcFbgB65Osa5+53FHx6XQic5O5mZocAvwU+JqvyvwL6uPt8\nMzsTGA10JtswRgIHAz8DVppZD+ACmjaKOwzA3f+Y8trydUwxs18BfcxsO+BiYBGw3N1HmNmRZKO5\nLoADJ7j7YjPbCpgK9AZeAlblbat7X3q5+wIzuzB/rWuA+939Z8CTwDZmNhfYA1hRsPyPgVFkHeJV\n4DR3/8DMbgNeAwYBffL7jnD3z8zsF8BrdZ/gDbxnmwKXAgemvmdSVf3iUWCau9dtf88Au6e+Pnd/\n3MzeAgaY2d+BR8gKz+7uPiQvshOB7sD7wPHu/ma+nd0C7A3MI9tm696z+cBwd/+LmZ2cv0frgKeA\n08hG0d3zfjGUrJ/ULX8cMC5/r98FRrr7PDMbD2wG7ED2QbKIbKCw0MxGA1u4+yXRy9sof/zsPM8B\ndjCzru7+SbH3I3VOdCLZH2lX4HTgxnwXtM7afEPpSPbpd7K770b2h+wMYGYHABcB+5N9aq4ALnH3\nacAMYKK7n+vua929bxN2a39OVhCSmVkHsjd1ZX5Tf+DqvIDuQjbaPcbde5P9USfly10AzHf3nYAx\nZH/0eN37k40K+gFfAb5lZt8DTgHm5SPitQXLD8zXNSi/bwEwvmCVw4GjyXYvvwB8FyB/7xosoLnT\ngEfd/c1GlpOmqYp+4e5Pu7vn6+tI9iH5dIVeY0f+3S+2Af4vL6CbA/cDP3P3L5H1ian5ciOBnmSD\ni+HAQfFK82m1CWQDgr7AFsBZZP1iVf5a3y5YfifgWuC7eb94OM91jgbOJusXS4CTANz96iIFFHdf\nlr/HdQ4GXmmogEJ6ET0E+J/837PJRmTbFNz/QP7/vkAHd//fPP+abDQFcBhwu7svcPd1wHXAkc1p\njJl9G1jp7k805/HRujoAZwJvAP/Ib17m7o/n/z4Y+JO7v5rna4Ej8scNBu4EcPc3KL7bNQx4wN0/\ncfeVZBvN9BJNOgS4s6Cz3EBYnB9w94/cfTXwN7JP36a8zg2BH5ONgqQyqq1fdMgf/wZwT3PWEa3v\nULJR9l/ymzoBdYVnP7JBQN186W3A7vme3GDgnrzwfwDMLLL6ocDs/HV/DhxDNjpvyFCyfljXR28A\nhphZXW2b5e7v5O/h8zSxX+Svc0/gv4EzSi2XtDtPVgguNLMtgc/z2woL8+L8/1sAHxXc/m7Bv7sD\nh5nZsILHd2pme74P3N7Qnflk9LbAGnf/cpFFOuW7C5BtzC+R7RavMzP49+upa/e3CpYHWEb2WnsA\nSwtuL3ztdbbk38WZurnI/HmK2apw+XydWxfkwudbS/bFWFMMBBa7+9xGl5Smqpp+kY9AbyHbrR2e\nF6Z4mSnA1/K4n7svLLKq2WZW98XSPOBgd1+eb68r3X15Qbst6hefkm2/xfpF4TYMWb9YUhfcfUXe\nxoZe4laE7+FHZHWtR56b1S/yPb+pZNMus0st2+wiamYbA3eRFZmH8gnZhr6U+BjoWpB7Ffz7PeBG\ndz+/uW3J29OBbATwXw0t04TJ6OCLpUa8BzwYf6mVt+UjsiME6mxV5PEfkG0wdY/pSTYH1JCFZLtC\ndXrmt6U6FPhDBdYjVF+/AG4iK3xHNPTlqLuf0IT1/OuLpUa8B/zN3b8R31FGv9ir4DGbk09xNGAh\n8NWC3JNs3nlx8cUbl49A7yCbqmt0rzZld74bsDHwbF7ARpN9gdK1yLJzgS55dYdsnqjO/cDwvIhg\nZkea2dj8vtVkn2xN0YvsD/RGWa+i+R4E9jezHQHMbB8zm5jf9xTwvfz2XYBvFnn8dLLd/83z0cIM\nsjmr1UDXaA4Nsl2f4WbWI3+/T6f47lC59iD7Ikoqo2r6hZkdQzYX+MNKHF3SRE+RfRHTP2/Dl8zs\nlvy9eAo43Mw2MLOtyabEYjOBwWa2Q/6Y64ETyV7zRmbWJVr+YeAAM6v79vwMssFNvRF3U+TTALcC\no5o6LdjsIprPaUwEXgCeI/t2egbwR6JPjnxI/iPgNjN7nmw3GWCduz8D/JJsd+FV4Bz+PTc4HfhP\nM5tqZhua2dx8F6mY7YFF+dxHi3P3+WR/sOl5u68inwclGw3vYmbzyObG7i3y+DnA1cCLZN+u/oVs\nBPNXsmPTFpB9QVS3/JNk85ZzyIrepmRfPJRkZr8ws9NKLLJ9/lxSAVXWL04nK6J/y5eZa2aNfcmY\nJN+tPwa4Nm/33cBdeb/8LdmofB7Ztl6sX7xF9p48RvberSTrJ/PJvhSbb2Z7R8ufATyQP98+ZN9l\nlGRmoy07TjW2L7AbcGXBezbXzPZoaF0d1sf5RPMh+hKga8Fciki7pn5Rm1rtZ59m9ryZHZXHY8nm\nTbShSLumflH7Ur+dL8cY4FdmdjnZp+3JrfjcItVK/aLGrZfdeRGRtkJncRIRSZCyO68hbO3r0Pgi\nUgb1idpXdp/QSFREJIGKqIhIAhVREZEEKqIiIglUREVEEqiIiogkUBEVEUmgIioikkBFVEQkgYqo\niEgCFVERkQSteSo8EalhK1asqHfbokWLgnzTTTcF+YUXXgjy/fffH+QrrrgiyD/96U+DvMEG1T/O\nq/4WiohUMRVREZEEKqIiIglSzmyvcyfWPp1PtLLaVJ945ZVXgnzyyfWvXPLcc8+Vtc643nToEG6C\nw4cPD/Lvfve7IG+22WZlPV8z6HyiIiKtSUVURCSBiqiISALNibZvmhOtrJruE6+99lqQv/rVrwa5\n2HGijdlkk02CHNebxtYZt+Gxxx4L8qabblp2mxqhOVERkdakIioikkBFVEQkgeZE2zfNiVZWTfWJ\nBQsWBHn33XcP8pIlS4IcH9MJ0KlTpyCffvrpQY5/Cx8799xzg3znnXeWXP7FF18M8m677VZy+WbQ\nnKiISGtSERURSaAiKiKSoGbPJ/rOO+8EecCAAUGO53v22muvIF955ZVB7t27dwVb1zQ9e/YMcgsc\n8ybSoKVLl5bMsWLzj7/+9a+DPHjw4LLacPbZZwe5sTnRu+66K8gXX3xxWc/XEjQSFRFJoCIqIpJA\nRVREJEHNHic6bty4IE+YMCHIjZ23sNz7K7GO+P5jjz02yFOmTKn3nC1Mx4lWVk0dJxqbPXt2kD/8\n8MMgDxkypN5junXrlvScTz75ZJAHDRpUcvm99947yE899VTS8xeh40RFRFqTiqiISAIVURGRBFV7\nnOiqVauCfNZZZwX5hhtuCHJ8ferG5npT76/UOkSqRWPzkS0h/p6g2O/zCz3zzDMt2Zxm0UhURCSB\niqiISAIVURGRBFU7J7pw4cIgT548OcjxHGg8l7LNNtsE+dRTT01uU3xc58svvxzkqVOnBnn69OlB\njtt4wgknJLdJpJaVe92miy66qIVa0nwaiYqIJFARFRFJoCIqIpKgan87/+mnnwZ5zz33DHJ8jex4\njnTGjBlBHjZsWAVbV9yYMWOCfM011wQ5nhNdu3Zti7epEfrtfGXpwOBGxMd/H3jggUGOf0sfe/fd\nd4Mcf/dRAfrtvIhIa1IRFRFJoCIqIpKgao8Tja83tP322wf5jTfeCHJjv7ltCU8//XSQb7rppiDH\nbYqv8yTS3px55plBbmwO9Ljjjgtyly5dKt6mVBqJiogkUBEVEUmgIioikqBq50RjY8eODfKsWbNK\nLh9f070lxOc0Xb58eZDjed3f/OY3Ld4maT8WL14c5Dlz5gT5jjvuCHL8vcImm2xScv1nnHFGkHv0\n6FFvmU6dOgU5Pg40ngONvzeIj+/eeeedgxwfa921a9cSLV4/NBIVEUmgIioikkBFVEQkQdX+dj4W\n/5b+8ssvD3K/fv2CfPTRR1e8DYsWLQpyr169ghwfFzp48OAgP/LIIxVvUyL9dr6yKton1qxZE+R7\n7rknyKeffnqQly1bFuTUY6fj2jBq1Kh6y4wYMSLIN998c5BvvPHGkus88sgjg3z99dcHuXv37k1q\nawXpt/MiIq1JRVREJIGKqIhIAhVREZEENfPFUjUo96TL48ePD/IFF1zQMg1rPn2xVFlJfeLtt98O\n8v7771/y/li3bt2CPHTo0CA/8cQTJR///vvvBzn+YqsSJ/np3bt3yTZttdVWyc+RSF8siYi0JhVR\nEZEEKqIiIgk0J1qGb33rW0F+7LHHghzPGX388cdBjk9IUgU0J1pZZfWJN998M8j77rtvkBcuXFjy\n8fFJec4999wgFzthSCkDBgwI8rPPPhvkSsyJvvrqq0HeZZddktdZYZoTFRFpTSqiIiIJVERFRBLU\nzEmZW9uLL75Y77Z4DjSeT77wwguDXIVzoFJF4pN1lDsHGh+HvNFGpbvz6tWrg3zttdcG+aWXXir5\n+Eq49dZbg3zppZe2+HO2NI1ERUQSqIiKiCRQERURSaA50QZcdNFF9W5r7Di5+MTQIqU8+OCDQY7n\n2LfddtsgT5gwoeT6PvnkkyC7e5DPP//8IDd2kvDPP/88yH369Km3TDzvX+y7hEKXXXZZkOOL5xU7\n8XO100hURCSBiqiISAIVURGRBPrtfC6+EF7Xrl3rLRPPie60005BjueDauA4Uf12vrLK6hMjR44M\ncnzcaKdOnYI8bNiwkut77rnngvzOO++U05x67r777iB/5zvfqbdMx44dgzxu3Lgg//KXvwxyXG/i\niz2+/vrrQe7cuXPTGls5+u28iEhrUhEVEUmgIioikkDHieauvvrqIBc7JjS+Lb5mUg3MgUoVKXbc\nZaFVq1YF+b777iu5fDzfGG+vO+64Y5Dj3+IPHz48yFtuuWXJ5ysm/j1/fA2lOC9YsCDIP/7xj4Mc\n/76/GmkkKiKSQEVURCSBiqiISIJ2e5zo0qVLg7zrrrsGOZ6rgfpzTP/85z+DvPXWW1eoda1Gx4lW\nVll9YuXKlUE+77zzgvzQQw8F+bXXXgvyDjvsEOQDDzwwyKecckqQ+/fvH+TGzj9aCYsXLw5yfF35\nuE/F5wuYP39+yzSsYTpOVESkNamIiogkUBEVEUnQbudE498Vx7+DL/a+7LXXXkF+5plnKt+w1qU5\n0cqqaJ+Iz+ewbNmyIHfp0iXIxc73UG3OOeecIE+aNCnIM2fODPJBBx3U4m2KaE5URKQ1qYiKiCRQ\nERURSaA50Vz8u+L4+jJQ/1yJ8fViapDmRCurpvuEAJoTFRFpXSqiIiIJVERFRBK02/OJTps2Lcjx\nb3g32ECfLyLSOFUKEZEEKqIiIglUREVEErTbOdFFixYFOT5etthxoqNGjWrRNolI7dFIVEQkgYqo\niEgCFVERkQTtdk70K1/5SpDj40R79epV7zGbbbZZi7ZJRGqPRqIiIglUREVEEqiIiogkaLfnExVA\n5xOtNPWJ2qfziYqItCYVURGRBCqiIiIJUo4T1XyaSEh9oh3SSFREJIGKqIhIAhVREZEEKqIiIglU\nREVEEqiIiogkUBEVEUmgIioikkBFVEQkgYqoiEgCFVERkQQlfztvZtcCB+RxZ+A94LM8f93dl7Vg\n2zCzXYGe7j6nCcsOByaQfTA8C5zi7p+U8VwbAauB14G1+XqWAOe5+6PNaH7hui8Btnf3kWY2Cxjt\n7i+UWP40d78+/3ejyye0a3PgRmBv4FPgfHe/r9LP09bUSr8ws47AVcAQsu35T8DZ7r62jOdqj/1i\nEHAl0A1YDowp9V6XLKLufmbBit8EftCUglZBRwFrgMY2li8B1wADgXn5v4cBdzbjOQe5+4J8vYOB\naWbWx90/bMa66nH3/Uvdn2/4VwDXN2X5RFcBb7n7cDPbDfiVmc0op5O1R7XSL4CfAjsB/chOjjIL\nGAFMbsZztot+YWabAPcBR7j7bDP7LjAF+GJDj0m62mf+iXgD0CNf1zh3v6Pg0+tC4CR3NzM7BPgt\n8DFZlf8V0Mfd55vZmcBooDPZhjESOBj4GbDSzHoAFwAvAwPd/YOoKT8E7nT3f+T57JTXVcfdHzez\nt4ABZvZ34BGyN3h3dx+Sb0wTge7A+8Dx7v6mmW0K3EI2wpsHvFbwns0Hhrv7X8zs5Pw9Wgc8BZxG\nNlrobmZzgaHAkwXLHweMI3uv3wVGuvs8MxsPbAbsQNZhFgGHu/tCMxsNbOHulxS+tnxjOZp843D3\nV8hGLJKoivrFo8A0d1+Vt+sZYPfU19eW+0W+jpHuPjvPc4AdzKxrQ3u2qXOiE8n+SLsCpwM3mtmG\nBfevzTeUjmSffie7+25kf8jOAGZ2AHARsD/Zp+YK4BJ3nwbMACa6+7nuvtbd+xbZUAD2ANaY2Z/M\n7O9mNikvEpXQEViZ/3sb4P/yDWVz4H7gZ+7+JWASMDVfbiTQE+gNDAcOileaj54nAIOAvsAWwFnA\nKcCq/LW+XbD8TsC1wHfdvS/wcJ7rHE324bEz2e7WSQDufnWRDQXAgGXAKDN7xcyezv8Wkq4q+oW7\nP+3unq+vI3Ag8HSFXmOb7Bfuvix/j+scDLxSamowtYgeAvxP/u/ZQBeyN7TOA/n/+wId3P1/8/xr\n/n3uxcOA2919gbuvA64DjiyzHd3JNpDjgb3y5zuvzHXUY2aHko0m/pLf1Amoe4P3A+YVzAvdBuxu\nZtsBg4F78g38A2BmkdUPBWbnr/tz4BiyUUhDhgJ/Khht3wAMMbO6v+Esd38nfw+fJ/v0LaU7sCWw\nNO/AlwL3mln3Rh4njauWfgGAmXXIH/8GcE9z1hGtry33i8LXuSfw38AZpZZL2p0nm3e80My2BD7P\nbysszIvz/28BfFRw+7sF/+4OHGZmwwoe36nMdiwFHnX39wHM7DpgDHBJ4UJmNgX4Wh73c/eFRdY1\n28zqJtDnAQe7+3IzA1jp7ssL2m357kWdT4GtyDawpQW3fwRsHT3PlmSfjAC4+4q8jQ29xq0I38OP\nyP5+PfJc+HxrgcKRTzFLyTrsb/Pnn2lm/yTb1Xq4kcdKadXSL+pGoLeQ7dYOzwtTvIz6RcTMBpKN\noE8q2LUvqtlF1Mw2Bu4im4B9KN99/rSBxT8GuhbkXgX/fg+40d3Pb25bgLeAzQvy2vy/gLuf0IR1\n/WsCvRHvAX9z92/Ed5jZR1F7tiry+A/IRs11j9mcfFeuAQuBrxbknmTza4uLL96ot8k6RBeyvw80\n8L5J01VZvwC4iezvfIS7rym2gPpFvXbuCdwBHOPuTzS2fMrufDdgY+DZfHdhNLCKcKOoMxfokld3\nyOaJ6twPDDezngBmdqSZjc3vW032ydaYO4DjzWy7fO7pFLKJ6Jb0FNmEc3/I5nLM7Jb8vXgKONzM\nNjCzrcnmVWIzgcFmtkP+mOuBE8le80Zm1iVa/mHgADOr+5bwDODBYiOLpsi/Vf0zMDZv/zeBL5Ad\nHibNVzX9wsyOIZsL/GFDBbQF1HS/yKcBbgVGNaWAQkIRzec0JgIvAM8BTjbh/UeiT458SP4j4DYz\nex54Kb9rnbs/A/ySbHfhVeAcYHp+/3TgP81sqpltaGZz812kuC1PAP9F9kd6Ffhnvs4Wk+++HANc\nm7f7buCufO7lt2Sjj3lko5J7izz+LbL35DGy924lcDUwn2zyf76Z7R0tfwbwQP58+wBnxuuNmdlo\ny47HK+ZkYKBlh+lMIvvkXdrAstIE1dQvyIryzsDf8mXmmtn1lXy9sTbQL/YFdgOuLHjP5prZHg2t\nq8O6desae76Ky4foS4CuBXMpIu2a+kVtarWffZrZ82Z2VB6PJZs30YYi7Zr6Re1L/Xa+HGPIfhFz\nOdmn7cmt+Nwi1Ur9osatl915EZG2QmdxEhFJkLI7ryFs7evQ+CJSBvWJ2ld2n9BIVEQkgYqoiEgC\nFVERkQQqoiIiCVRERUQSqIiKiCRQERURSdCaP/useRdddFGQx44dG+TNNtusNZsjIlVAI1ERkQQq\noiIiCVRERUQStNs50dWrV5e8/xe/+EW928aPHx/k3XcPL+F97LHHpjdMRGqKRqIiIglUREVEEqiI\niogkaLdzokuWLAnyiBEjgjx//vx6j+ndu3eQ+/btW/mGibSShQsXBvn+++8P8quvvhrkq666quzn\n2G677YI8adKkIB9++OFlr7PaaCQqIpJARVREJIGKqIhIgpSrfdb09WROPPHEIN96662NPubll18O\n8q677lrRNq0HusZSZVVVn3jvvfeC3L9//yC///77QV67dm2Lt6lDh3CTmzVrVpAHDRrU4m1ohK6x\nJCLSmlRERUQSqIiKiCRos8eJLl++PMjxMW+33357ycdPmTKl3m1tYA5U2rAHH3wwyEceeWSQV6xY\nkbT+HXfcsd5tp556asnHxPOu11xzTZD//Oc/B7kK5kTLppGoiEgCFVERkQQqoiIiCdrscaLDhw8P\n8r333lty+XiO9Hvf+169ZTp16pTesOqi40Qra732ifiaXxMnTiy5fDzHf+mllwZ52LBhQd5gg/pj\nro033rjkcyxbtizIAwYMCHJcf+LvLtYDHScqItKaVERFRBKoiIqIJGgzx4l+9NFHQX7kkUdKLv/9\n738/yPEcaBuc/5Q2rl+/fiXvj+dAJ0yYEOTDDjus4m3q1q1bkONrl8X9bs6cOUEeOHBgxdtUaRqJ\niogkUBEVEUmgIioikqDNzIl++umnQY6vobT11lsHefLkyUHu2LFjyzRMpJV079695P0PP/xwkL/w\nhS+0ZHOKmjlzZpDjc5im/r5/fdBIVEQkgYqoiEgCFVERkQQ1Oyc6d+7cIB966KFB3nDDDYM8Y8aM\nIGsOVNqaoUOHBvmxxx4L8hZbbNGazWk3NBIVEUmgIioikkBFVEQkQc3OiX722WdB/sc//hHk+LjQ\nr3/96y3epr/+9a9BXrx4ccnld9555yB/8YtfrHibpP3o3LlzkGvxekW1SCNREZEEKqIiIglUREVE\nEtTsnGhj4uvDVMLjjz8e5EMOOSTI8bXuY/H1ZDbZZJMg//73vw/yEUccUW4TRarKhx9+GOS77747\nyJtuummQt9122xZvU6VpJCoikkBFVEQkgYqoiEiCmpkTjc87eMMNN5Rcfty4cUnPd9ZZZ9W7bdKk\nSUHu0CG8RHWPHj2CvP322we5T58+QY7nh8aPHx/k+Jo38fkARKrNypUrg3zxxRcHOZ4j3WWXXYL8\n5S9/uWUa1oI0EhURSaAiKiKSQEVURCRBh/jYxTI0+4HNsXr16iBvvPHGQd5gg/Dz4J133glyr169\nSq4/Pj/pvvvuW2+Z+Nr28e/z77rrriDHv11+9913g/wf//EfJdsUzy+1wDlQOzS+iJShVftENXr6\n6aeDvM8++wQ5Pi506tSpQY7PC7welN0nNBIVEUmgIioikkBFVEQkgYqoiEiCmjnYPj6wPf5SpkuX\nLkGOD3yPxSdQjk/2EX+JBPDNb34zyLfffnvJNom0dS+99FKQDz/88JLLX3PNNUGugi+SkmkkKiKS\nQEVURCSBiqiISIKamRPdaKOwqZdddlmQTzrppCCfeOKJQb755puDHM9vrlixIsjHH398vTbccsst\nJdsU/3AhPsHImDFj6q2z0NixY0uuX6Q1xScLiS8GCfVPkrNo0aIgxyceHzBgQIVaVz00EhURSaAi\nKiKSQEVURCRBzZyAJBYfn7bnnnsGOT4hyY477hjk119/PcjxCZTj+6H+SZHjk5b8/Oc/D3I8JxrP\ncW633XZBjk/esM0229RrQ4XpBCSVVVUnIInn+Z977rkg33bbbSUfH2+/8RxpU/Tr1y/I8TYen0io\nCugEJCIirUlFVEQkgYqoiEiCmp0Tjf3oRz8K8nXXXVfW4+OTxZ5//vn1lpkzZ06QH3744bKeIz5p\n81FHHVXW41uA5kQra732ifgk3vGx0nfeeWdrNqeo+LjSvfbaK8ijR48u+fjOnTuXzBWgOVERkdak\nIioikkBFVEQkQZuZE12+fHmQTzvttCDHF8RqCfG5FC+++OIg77HHHkGOz5G6Hqz3BrQxrdonVq1a\nFeQf/OAHQY6P80xV7Ljl+DjP+Lfz8bGq5YqP9545c2aQDzrooKT1F6E5URGR1qQiKiKSQEVURCRB\nmzlhZXyNpfjcn/F8Tnytl6bMDcfLjBgxIshXXHFFkLfddttG1ynSXNOmTQtypedAv/a1rwX50Ucf\nrbdM165dg/yHP/whyKecckqQ16xZE+TFixeXbMNPfvKTILfAHGgyjURFRBKoiIqIJFARFRFJ0GaO\nEy3XN77xjSC/8sorQR41alS9xwwZMiTI3/72t4Ncg9dE0nGildWqfeK4444Lcrm/jY/nMy+99NIg\nDxw4MMj9+/cva/3FxMeNPvDAAyWX33fffYPcq1ev5DY0QseJioi0JhVREZEEKqIiIgna7ZyoAJoT\nrbRW7RPdu3cP8scff1xy+XgftJMmAAAA00lEQVROf/r06UGOrxHfTmlOVESkNamIiogkUBEVEUlQ\ncwc2ikgmvkbX5MmTg7zffvsFecaMGUFugesTtUsaiYqIJFARFRFJoCIqIpJAx4m2bzpOtLLUJ2qf\njhMVEWlNKqIiIglUREVEEqiIiogkUBEVEUmgIioikkBFVEQkgYqoiEgCFVERkQQqoiIiCVRERUQS\nqIiKiCRQERURSaAiKiKSQEVURCRByjWWdC5KkZD6RDukkaiISAIVURGRBCqiIiIJVERFRBKoiIqI\nJFARFRFJ8P8r8QuQgglWKwAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 4 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    }
  ]
}